# -*- encoding: utf-8 -*-
# author:lmolhw
# datetime:2020-11-30 10:45

"""
文件说明：清洗new_train_lebeled.txt
"""

def open_txt(file_name):
    with open(file_name, 'r') as f:
        try:
            line = f.readline()
            while line:
                yield line.strip()
                line = f.readline()
        except:
            print('No value')
def conver_label(gt):
    cn = ['！', '（', '）', '：', ',', '．', '［', '］', '；', '％', '０', '１', '２', '３', '４', '５', '６', '７', '８', '９', '％', '－',
          'Ａ', 'Ｂ', 'Ｃ', 'Ｄ',
          'Ｅ',
          'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', 'Ｕ', 'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ',
          'ｂ',
          'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', 'ｋ', 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｗ',
          'ｙ',
          'ｚ', '￥', ' ']
    en = ['!', '(', ')', ':', '，', '.', '[', ']', ';', '%', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '%', '-',
          'A', 'B', 'C', 'D',
          'E',
          'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a',
          'b',
          'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
          'y',
          'z', '¥', '']
    gt = ''.join(gt.split()).strip()
    gt_tmp = list(gt)

    for i in range(len(gt_tmp)):
        c = gt_tmp[i]
        if c in cn:
            n = cn.index(c)
            gt_tmp[i] = en[n]

    # for i in range(len(gt_tmp)):
    #     for c in range(len(cn)):
    #         if gt_tmp[i] == cn[c]:
    #             gt_tmp[i] = en[c]

    gt_tmp = ''.join(gt_tmp)
    '''
    for i in range(len(gt_tmp)):
        if c in cn:
            j = cn.index(c)
            gt_tmp[i] = en[j]
    '''
    return gt_tmp

org_txt = '/data2/dtj/DTJCOMP/ppccddff/train_labeled.txt'
new_txt = open('/data2/dtj/DTJCOMP/ppccddff/labeled_new.txt', 'w')
for line in open_txt(org_txt):
    file = line.split('\t')[0].strip()
    gt = line.split('/')[-1].split('\t')[0][:-6]
    gt = conver_label(gt)
    # print(gt)
    file = file[:7] + 'dtj/' +file[7:]
    if gt:
        new_txt.writelines(file + '\t' + gt + '\n')
    else:
        print(line)
